﻿using System;
using System.Collections.Generic;
using System.Text;
using System.Net;
using System.IO;
using System.Text.RegularExpressions;
using System.Web;
using System.Threading;
using System.Collections;

namespace Spider
{
    public enum Status { STATUS_FAILED, STATUS_SUCCESS, STATUS_UNQUEUEDING, STATUS_QUEUEDING };

    public class PageClass
    {
        public string html;
        public Uri URL;
    }

    public class Util
    {
        private static string RegExprHREF = @"(?<=a\s+([^>]+\s+)?href\=[\x27\x22])(?<1>[^\x27\x22]*)(?=[\x27\x22])";
        //private static string RegExprHREF="<a[^>]+href=[\"']?(?'url'[^\"']+)[\"']?[^>]*>";
        private static Regex RegExFindHref = new Regex(RegExprHREF, RegexOptions.Singleline | RegexOptions.IgnoreCase | RegexOptions.Compiled);
        public Dictionary<Uri, Status> DicProductUrl = new Dictionary<Uri, Status>();
        public Queue QueueProduct = new Queue();
        public Dictionary<Uri, Status> DicQueryUrl = new Dictionary<Uri, Status>();
        public Queue QueueQueryUrl = new Queue();
        public Queue UnParserProductUrl = new Queue();
        public bool AbortTag = false;
        public bool Pause = false;
        public Done done = new Done();
        public bool isFinishSpider = false;
        public MonitorManager mMonitorManager = null;

        public List<CrawWorker> clist = new List<CrawWorker>();
        public List<ScrapWorker> slist = new List<ScrapWorker>();

        frmMain mInfoForm;

        private Thread thread;

        public void clear()
        {
            
            isFinishSpider = false;
            mMonitorManager = null;
            Pause = false;
            AbortTag = false;
            UnParserProductUrl.Clear();
            DicProductUrl.Clear();
            QueueProduct.Clear();
            DicQueryUrl.Clear();
            QueueQueryUrl.Clear();
            Product.Instance.Clear();
            done.Reset();
        }

        #region Properties
        public frmMain InfoForm
        {
            get
            {
                return mInfoForm;
            }
            set
            {
                mInfoForm = value;

            }
        }

        public int ProductCount
        {
            get
            {
                return Product.Instance.ProductList.Count;
            }
        }

        public int WaitUrlCount
        {
            get
            {
                return QueueQueryUrl.Count;
            }
        }

        public int UrlCount
        {
            get
            {
                return DicQueryUrl.Count;
            }
        }


        public int WaitProductUrlCount
        {
            get
            {
                return QueueProduct.Count;
            }
        }

        public int ProductUrlCount
        {
            get
            {
                return DicProductUrl.Count;
            }
        }
        #endregion

        public void defaultOper()
        {
            this.addQueryURI(new Uri(DataHandler.Instance.CurrentRule.UrlPrefix));
        }

        private string ErrorMsg(string URL, string Msg)
        {
            return String.Format("Error Message:{0}    Request URL:{1}  DateTime:{2}", Msg, URL, DateTime.Now.ToString("MM/dd/yyyy hh:mm:ss"));
        }

        #region obtainUrl



        public List<Uri> GetHrefs(string currentURL, string htmlData)
        {
            if (AbortTag) return null;//stop
            List<Uri> listUrl = GetEnumerator(RegExFindHref, currentURL, htmlData);

            for (int i = 0; i < listUrl.Count; i++)
            {
                if (AbortTag)//stop
                    break;

                if (listUrl[i] == null)
                    continue;

                if (!listUrl[i].ToString().ToLower().Contains(DataHandler.Instance.CurrentRule.UrlPrefix.ToLower()))
                    continue;

                if (listUrl[i].ToString().ToLower().Contains(DataHandler.Instance.CurrentRule.ProductUrlFilter.ToLower()))
                {
                    addProductURI(IgnoreCase(listUrl[i])); 
                }

                addQueryURI(IgnoreCase(listUrl[i]));
            }
            return listUrl;
        }

        private List<Uri> GetEnumerator(Regex regExpr, string currentURL, string html)
        {
            List<Uri> listUrl = new List<Uri>();
            Uri uri = null;
            Match n = regExpr.Match(html);

            for (Match m = regExpr.Match(html); m.Success; m = m.NextMatch())
            {
                if (AbortTag) break;//stop
                string href = m.Groups[1].ToString();

                // filter non-real relation urls:
                if (String.IsNullOrEmpty(href) ||
                    href.StartsWith("#") ||
                    href.StartsWith("mailto:") ||
                    href.StartsWith("javascript:"))
                {
                    continue;
                }

                href = System.Web.HttpUtility.HtmlDecode(href);

                try
                {
                    if (!href.ToLower().StartsWith("http://"))
                    {
                        if (href.IndexOf('/') == 0)
                            uri = ConvertToAbsoluteUrl(href, "http://" + (new Uri(currentURL).Host));
                        else
                        {
                            int lastIndex = currentURL.LastIndexOf('/');
                            uri = ConvertToAbsoluteUrl(href, currentURL.Substring(0, lastIndex + 1));
                        }
                    }
                    else
                        uri = new Uri(href);

                    //Log.Instance.WriteMsg(string.Format("href: {0}   CurrentURL: {1}     URL: {2}", href, currentURL, uri.ToString()));


                    if (uri != null)
                        listUrl.Add(uri);
                }
                catch (Exception ex)
                {
                    Log.Instance.WriteException(ErrorMsg(uri.ToString(), ex.Message));
                }
            }
            return listUrl;
        }

        private Uri ConvertToAbsoluteUrl(string url, string baseUrl)
        {
            // we try to prevent the exception caused in the case the url is relative
            // (no scheme info) just for speed
            if (url.IndexOf(Uri.SchemeDelimiter) < 0 && baseUrl != null)
            {
                try
                {
                    Uri baseUri = new Uri(baseUrl);
                    return new Uri(baseUri, url);
                }
                catch (Exception ex)
                {
                    Log.Instance.WriteException(ErrorMsg(url.ToString(), ex.Message));
                    return null;
                }
            }

            try
            {
                Uri uri = new Uri(url);
                return uri;
            }
            catch (System.UriFormatException ex)
            {
                Log.Instance.WriteException(ErrorMsg(url.ToString(), ex.Message));
                return null;
            }
            catch (Exception)
            {
                if (baseUrl != null)
                {
                    try
                    {
                        Uri baseUri = new Uri(baseUrl);
                        return new Uri(baseUri, url);
                    }
                    catch (WebException ex)
                    {
                        Log.Instance.WriteException(ErrorMsg(url.ToString(), ex.Message));
                        return null;
                    }
                    catch (Exception ex)
                    {
                        Log.Instance.WriteException(ErrorMsg(url.ToString(), ex.Message));
                        return null;
                    }
                }
                else
                {
                    return null;
                }
            }
        }
        #endregion

        #region Crawl
       

        private void addQueryURI(Uri uri)
        {
            Monitor.Enter(this);
            if (!DicQueryUrl.ContainsKey(uri))
            {
                DicQueryUrl.Add(uri, Status.STATUS_UNQUEUEDING);
                QueueQueryUrl.Enqueue(uri);
                showMsg();

            }
            Monitor.Pulse(this);
            Monitor.Exit(this);
        }

        public void modifyQueryURIStatus(Uri URL, Status MStatus)
        {
            Monitor.Enter(DicQueryUrl);
            DicQueryUrl[URL] = MStatus;
            Monitor.Exit(DicQueryUrl);
        }

        private void addProductURI(Uri uri)
        {
            Monitor.Enter(this);
            if (!DicProductUrl.ContainsKey(uri))
            {
                DicProductUrl.Add(uri, Status.STATUS_UNQUEUEDING);
                QueueProduct.Enqueue(uri);
                showMsg();
            }
            Monitor.Pulse(this);
            Monitor.Exit(this);
        }

        public void modifyProductURIStatus(Uri URL, Status MStatus)
        {
            Monitor.Enter(DicProductUrl);
            DicProductUrl[URL] = MStatus;
            Monitor.Exit(DicProductUrl);
        }


        private Uri IgnoreCase(Uri uri)
        {
            string url = uri.ToString();
            Regex m_regex = new Regex(@"[?|&]{1}([\w]+])=");
            Match match = m_regex.Match(url);
            string temp;
            while (match.Success)
            {
                temp = match.Groups[0].Value;
                if (temp != string.Empty)
                    url = url.Replace(temp, temp.ToLower());
                match.NextMatch();
            }
            return new Uri(url);
        }
        #endregion

        #region multithreading
        public void Init()
        {
            DataHandler.ResetDataHandler();
            Product.Reset();
            Rules.Reset();
            defaultOper();
            this.LoadProductList();
            this.LoadQueryURL();
            this.LoadProductURL();
            mMonitorManager = new MonitorManager(this);
        }

        public void startProcess()
        {

            //ThreadStart starter1 = new ThreadStart(this.LoadProductURL);
            //td[0] = new Thread(starter1);
            //td[0].Start();

            //ThreadStart starter2 = new ThreadStart(this.LoadProductList);
            //td[1] = new Thread(starter2);
            //td[1].Start();

            //ThreadStart starter3 = new ThreadStart(this.LoadQueryURL);
            //td[2] = new Thread(starter3);
            //td[2].Start();


            //done.WaitBegin();
            //done.WaitDone();
            //done.Reset();

            //this.isFinishLoad = true;//数据加载完毕
            uint crawWorkThreadNum = Spider.Properties.Settings.Default.CrawWorkThreadNum;
            uint scrapWorkerThreadNum = Spider.Properties.Settings.Default.ScrapWorkerThreadNum;

            for (int i = 0; i < crawWorkThreadNum; i++)
            {
                CrawWorker cworker = new CrawWorker(this, string.Format("CrawThread", i + 1));
                cworker.Start();
                clist.Add(cworker);
            }

            for (int i = 0; i < scrapWorkerThreadNum; i++)
            {
                ScrapWorker sworker = new ScrapWorker(this, string.Format("ScrapThread", i + 1));
                sworker.Start();
                slist.Add(sworker);
            }

            mMonitorManager.Start();

            //done.WaitBegin();
            //done.WaitDone();
            //AbortTag = true;

            //SaveInfos();//save everything after stop

            //if (isFinishSpider)
            //{
            //    clear();
            //    mInfoForm.clearForm();
            //    isFinishSpider = false;
            //    start();
            //}
        }

        public void LoadProductList()
        {
            //done.WorkerBegin();
            Product.Instance.LoadProductList();//加载产品
            //done.WorkerEnd();
        }

        public void LoadProductURL()
        {
            //done.WorkerBegin();
            DicProductUrl = DataHandler.Instance.LoadDicProURL();

            if (DicProductUrl.Count != 0)
            {
                foreach (KeyValuePair<Uri, Status> kvp in DicProductUrl)
                {
                    if (kvp.Value == Status.STATUS_UNQUEUEDING || kvp.Value == Status.STATUS_QUEUEDING)
                    {
                        QueueProduct.Enqueue(kvp.Key);
                        showMsg();
                    }
                }
            }
            //done.WorkerEnd();
        }

        public void LoadQueryURL()
        {
            //done.WorkerBegin();
            DicQueryUrl = DataHandler.Instance.LoadDicQueryURL();
            if (DicQueryUrl.Count != 0)
            {
                foreach (KeyValuePair<Uri, Status> kvp in DicQueryUrl)
                {
                    if (kvp.Value == Status.STATUS_UNQUEUEDING || kvp.Value == Status.STATUS_QUEUEDING)
                    {
                        QueueQueryUrl.Enqueue(kvp.Key);
                        showMsg();
                    }
                }
            }
            //done.WorkerEnd();
        }

        public void start()
        {
            ThreadStart starter = new ThreadStart(this.startProcess);
            thread = new Thread(starter);
            thread.IsBackground = true;
            thread.Start();
        }

        public void setPause()
        {
            foreach (CrawWorker cw in clist)
            {
                if (cw.MThread.ThreadState != ThreadState.Stopped)
                    cw.MThread.Suspend();
            }
            foreach (ScrapWorker sw in slist)
            {
                if (sw.MThread.ThreadState != ThreadState.Stopped)
                    sw.MThread.Suspend();
            }
            if (thread.ThreadState != ThreadState.Stopped)
                thread.Suspend();
        }

        public void reStart()
        {
            if (thread.ThreadState != ThreadState.Stopped)
                thread.Resume();
            foreach (CrawWorker cw in clist)
            {
                if (cw.MThread.ThreadState != ThreadState.Stopped)
                    cw.MThread.Resume();
            }
            foreach (ScrapWorker sw in slist)
            {
                if (sw.MThread.ThreadState != ThreadState.Stopped)
                    sw.MThread.Resume();
            }
        }

        public void Abort()
        {
            AbortTag = true;
            //if (Pause)
            //{
            //    Pause = false;
            //    reStart();
            //}
            foreach (CrawWorker cw in clist)
            {
                cw.MThread.Abort();
            }
            foreach (ScrapWorker sw in slist)
            {
                if (sw.MThread.ThreadState == ThreadState.WaitSleepJoin)
                    Monitor.Exit(sw);
                sw.MThread.Abort();
            }
        }

        public void showMsg()
        {
            //string info = string.Format("Product URLs Qty:{0}   Watting Product URLs Qty:{1}\r\nSearched URLs Qty:{2}Watting URLs Qty:{3}\r\nProducts Qty:{4}\r\nError URLs Qty:{5}\r\n",
            //    DicProductUrl.Count, QueueProduct.Count, DicQueryUrl.Count, QueueQueryUrl.Count, Product.Instance.ProductList.Count, UnParserProductUrl.Count);
            string[] infos = new string[6];
            infos[0] = string.Format("Product URLs Qty:{0}", DicProductUrl.Count);
            infos[1] = string.Format("Watting Product URLs Qty:{0}", QueueProduct.Count);
            infos[2] = string.Format("Searched URLs Qty:{0}", DicQueryUrl.Count);
            infos[3] = string.Format("Watting URLs Qty:{0}", QueueQueryUrl.Count);
            infos[4] = string.Format("Products Qty:{0}", Product.Instance.ProductList.Count);
            infos[5] = string.Format("Error URLs Qty:{0}", UnParserProductUrl.Count);
            InfoForm.showMsg(infos);
        }

        public void SaveInfos()
        {

            if (!isFinishSpider)
            {
                DataHandler.Instance.SaveDicQueryURL(DicQueryUrl);
                DataHandler.Instance.SaveDicProURL(DicProductUrl);
                DataHandler.Instance.SaveUnParserProductURL(UnParserProductUrl.ToArray());
            }
            else
            {
                DataHandler.Instance.ClearTempUrl();
            }
            Product.Instance.SaveProductList();
        }
        #endregion

        #region Scrapping

        public void addProduct(Product pro)
        {
            if (AbortTag) return;//stop;
            Monitor.Enter(DataHandler.Instance);
            Product.Instance.AddProduct(pro);
            Monitor.Pulse(DataHandler.Instance);
            Monitor.Exit(DataHandler.Instance);
        }

       
        #endregion


    }
}
